{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# for other action\n",
    "# construct input for tool 2 from tool 1's output\n",
    "folder_name_1 = '/Users/kavyasrinet/Github/minecraft/python/craftassist/text_to_tree_tool/turk_data/' + \\\n",
    "                'combined_dicts/humanbot/other_actions/'\n",
    "folder_name_2 = '/Users/kavyasrinet/Github/minecraft/python/craftassist/text_to_tree_tool/turk_data/' + \\\n",
    "                'combined_dicts/humanbot/other_actions/'\n",
    "\n",
    "# tool_1_out_file = folder_name_1 + 'all_agreements.txt'\n",
    "# tool2_input_file = folder_name_2 + 'input.txt'\n",
    "\n",
    "tool_1_out_file = folder_name_1 + 'all_other_actions.txt'\n",
    "tool2_input_file = folder_name_2 + 'all_other_tool_2_input.txt'\n",
    "from pprint import pprint\n",
    "'''\n",
    "command: Input.command\n",
    "\n",
    "'''\n",
    "import copy\n",
    "example_d = {}\n",
    "\n",
    "def merge_indices(indices):\n",
    "    a, b = indices[0]\n",
    "    for i in range(1, len(indices)):\n",
    "        x, y = indices[i]\n",
    "        if x != b + 1:\n",
    "            return indices\n",
    "            # raise NotImplementedError(\"Spans not continuous during merging!!!\")\n",
    "        else:\n",
    "            b = y\n",
    "    if a==b:\n",
    "        return a\n",
    "    return [a, b]\n",
    "\n",
    "import ast\n",
    "import json\n",
    "\n",
    "\n",
    "all_actions = {}\n",
    "with open(tool_1_out_file, \"r\") as f1, open(tool2_input_file, 'w') as f2:\n",
    "    composites= []\n",
    "    \n",
    "    for line in f1.readlines():\n",
    "        line  = line.strip()\n",
    "        cmd, out = line.split(\"\\t\")\n",
    "        words = cmd.split()\n",
    "        \n",
    "        words_copy = copy.deepcopy(words)\n",
    "        #pprint(out)\n",
    "        action_dict = ast.literal_eval(out)\n",
    "        action_type = action_dict['action_type'][1]\n",
    "            \n",
    "        # find children that need to be re-annotated\n",
    "        for key, val in action_dict.items():\n",
    "            child_name = None\n",
    "            # child needs annotation\n",
    "            if val[0]== 'no':\n",
    "                # insert \"span\"\n",
    "                words_copy = copy.deepcopy(words)\n",
    "                child_name = key\n",
    "                \n",
    "                write_line = \"\"\n",
    "                write_line += \" \".join(words) + \"\\t\"\n",
    "                #print(words, child_name, action_type, val[1])\n",
    "                \n",
    "                indices = merge_indices(val[1])\n",
    "                span_text = None\n",
    "                if type(indices) == list:\n",
    "                    if type(indices[0]) == list:\n",
    "                        # this means that indices were scattered and disjoint\n",
    "                        for idx in indices:\n",
    "                            words_copy[idx[0]] = \"<span style='background-color: #FFFF00'>\" + words_copy[idx[0]]\n",
    "                            words_copy[idx[1]] = words_copy[idx[1]] + \"</span>\"\n",
    "                    else:\n",
    "                        words_copy[indices[0]] = \"<span style='background-color: #FFFF00'>\" + words_copy[indices[0]]\n",
    "                        words_copy[indices[1]] = words_copy[indices[1]] + \"</span>\"\n",
    "                else:\n",
    "                    words_copy[indices] = \"<span style='background-color: #FFFF00'>\" + words_copy[indices] + \"</span>\"\n",
    "                write_line += \" \".join(words_copy) + \"\\t\" + action_type + \"\\t\" + child_name\n",
    "                f2.write(write_line+ \"\\n\")\n",
    "                #print(write_line)\n",
    "        \n",
    "        #print(\"----\"*5)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# combining for otheraction\n",
    "# output of tool 1\n",
    "folder = '/Users/kavyasrinet/Github/minecraft/python/craftassist/text_to_tree_tool/turk_data/combined_dicts/humanbot/other_actions/'\n",
    "tool_1_out_file = folder + 'tool1_out.txt'\n",
    "# output of tool 2\n",
    "tool_2_out_file = folder + 'all_agreements.txt'\n",
    "\n",
    "# combine outputs\n",
    "# check if all keys of t1 annotated yes -> put directly\n",
    "# if no , check child in t2 and combine\n",
    "# construct mape of tool 1\n",
    "tool1_map = {}\n",
    "with open(tool_1_out_file) as f:\n",
    "    for line in f.readlines():\n",
    "        line = line.strip()\n",
    "        cmd, a_d = line.split(\"\\t\")\n",
    "        tool1_map[cmd]= a_d\n",
    "\n",
    "# construct map of tool 2\n",
    "tool2_map = {}\n",
    "with open(tool_2_out_file) as f:\n",
    "    for line in f.readlines():\n",
    "        line = line.strip()\n",
    "        cmd, child, child_dict = line.split(\"\\t\")\n",
    "        if cmd in tool2_map and child in tool2_map[cmd]:\n",
    "            print(\"BUGGG\")\n",
    "        if cmd not in tool2_map:\n",
    "            tool2_map[cmd] = {}\n",
    "        tool2_map[cmd][child] = child_dict        \n",
    "\n",
    "import ast\n",
    "def all_yes(a_dict):\n",
    "    if type(a_dict) == str:\n",
    "        a_dict = ast.literal_eval(a_dict)\n",
    "    for k, val in a_dict.items():\n",
    "        if type(val) == list and val[0] == \"no\":\n",
    "            return False\n",
    "    return True\n",
    "\n",
    "def clean_dict_1(a_dict):\n",
    "    if type(a_dict) == str:\n",
    "        a_dict = ast.literal_eval(a_dict)\n",
    "    new_d = {}\n",
    "    for k, val in a_dict.items():\n",
    "        if type(val) == list:\n",
    "            if val[0] in [\"yes\", \"no\"]:\n",
    "                new_d[k] = val[1]                \n",
    "        elif type(val) == dict:\n",
    "            new_d[k] = a_dict(val[1])\n",
    "        else:\n",
    "            new_d[k] = val\n",
    "    return new_d\n",
    "\n",
    "i  =0\n",
    "from pprint import pprint\n",
    "# update dict of tool1 with tool 2\n",
    "with open(folder + 'final/other_actions_resolved.txt', 'w') as f:\n",
    "    for cmd, a_dict in tool1_map.items():\n",
    "        clean_dict = clean_dict_1(a_dict)\n",
    "        if all_yes(a_dict):\n",
    "            f.write(cmd + \"\\t\" + str(clean_dict) + \"\\n\")\n",
    "            #print(clean_dict)\n",
    "            continue\n",
    "        child_dict = tool2_map[cmd]\n",
    "        for k, v in child_dict.items():\n",
    "            if k not in clean_dict:\n",
    "                print(cmd, k)\n",
    "                print(a_dict)\n",
    "                print(clean_dict)\n",
    "                print(child_dict)\n",
    "                print(\"BUGGGG\")\n",
    "            if type(v) == str:\n",
    "                v = ast.literal_eval(v)\n",
    "            #print(k, v)\n",
    "            if not v:\n",
    "                continue\n",
    "            #print(cmd, a_dict, child_dict)\n",
    "            if k == \"tag_val\":\n",
    "                clean_dict.update(v)\n",
    "            else:\n",
    "                clean_dict[k] = v[k]\n",
    "        valid_dict = {}\n",
    "        valid_dict['dialogue_type'] = clean_dict['dialogue_type']\n",
    "        del clean_dict['dialogue_type']\n",
    "        #print(actual_dict['action_type'])\n",
    "        clean_dict['action_type'] = clean_dict['action_type'].upper()\n",
    "        valid_dict['action_sequence'] = [clean_dict]\n",
    "        f.write(cmd + \"\\t\" + str(valid_dict) + \"\\n\")\n",
    "#     pprint(clean_dict)\n",
    "#     print(\"*\"*20)\n",
    "#     if i==20:\n",
    "#         break\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'ast' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-1-b384da823207>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     46\u001b[0m         \u001b[0mline\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mline\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstrip\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     47\u001b[0m         \u001b[0mtext\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0md\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mline\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msplit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"\\t\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 48\u001b[0;31m         \u001b[0md\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mast\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mliteral_eval\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0md\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     49\u001b[0m         \u001b[0maction_dict\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfix_spans\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0md\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'action_sequence'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     50\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0maction_dict\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'action_type'\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;34m'TAG'\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mNameError\u001b[0m: name 'ast' is not defined"
     ]
    }
   ],
   "source": [
    "# post_process spans and \"contains_coreference\" : \"no\"\n",
    "def merge_indices(indices):\n",
    "    a, b = indices[0]\n",
    "    for i in range(1, len(indices)):\n",
    "        a = min(a, indices[i][0])\n",
    "        b = max(b, indices[i][1])\n",
    "    return [a, b]\n",
    "\n",
    "def fix_put_mem(d):\n",
    "    \n",
    "    if type(d) == str:\n",
    "        d = ast.literal_eval(d)\n",
    "    new_d = copy.deepcopy(d)\n",
    "    del new_d['action_type']\n",
    "    if 'has_tag' in new_d and 'upsert' in new_d:\n",
    "        new_d['upsert']['memory_data']['has_tag'] = new_d['has_tag']\n",
    "        del new_d['has_tag']\n",
    "\n",
    "    return new_d\n",
    "def fix_spans(d):\n",
    "    new_d = {}\n",
    "    if type(d) == str:\n",
    "        d = ast.literal_eval(d)\n",
    "    for k, v in d.items():\n",
    "        if k == \"contains_coreference\" and v == \"no\":\n",
    "            continue\n",
    "        if type(v) == list:\n",
    "            if k == \"tag_val\":\n",
    "                new_d[\"has_tag\"] = [0, merge_indices(v)] \n",
    "            else:\n",
    "                new_d[k] = [0, merge_indices(v)] \n",
    "            continue\n",
    "        elif type(v) == dict:\n",
    "            new_d[k] = fix_spans(v)\n",
    "            continue\n",
    "        else:\n",
    "            new_d[k] = v\n",
    "    return new_d\n",
    "        \n",
    "i = 0\n",
    "input_file = '../../minecraft/python/craftassist/text_to_tree_tool/turk_data/combined_dicts/humanbot/other_actions/final/other_actions_resolved.txt'\n",
    "output_file = '../../minecraft/python/craftassist/text_to_tree_tool/turk_data/combined_dicts/humanbot/other_actions/final/other_actions_postprocessed.txt'\n",
    "with open(input_file) as f, open(output_file, 'w') as f_w:\n",
    "    for line in f.readlines():\n",
    "        i+= 1\n",
    "        line = line.strip()\n",
    "        text, d = line.split(\"\\t\")\n",
    "        d = ast.literal_eval(d)\n",
    "        action_dict = fix_spans(d['action_sequence'][0])\n",
    "        if action_dict['action_type'] == 'TAG':\n",
    "            updated_dict = fix_put_mem(action_dict)\n",
    "            new_d = {}\n",
    "            new_d['dialogue_type'] = d['dialogue_type']\n",
    "            new_d.update(updated_dict)\n",
    "        elif action_dict['action_type'] == 'ANSWER':\n",
    "            #print(d)\n",
    "            new_d = {}\n",
    "            new_d['dialogue_type'] = 'GET_MEMORY'\n",
    "        else:\n",
    "            if action_dict['action_type'] == 'COPY':\n",
    "                action_dict['action_type'] = 'BUILD'\n",
    "            d['action_sequence'] = [action_dict]\n",
    "            new_d = d\n",
    "        f_w.write(text+  \"\\t\" + json.dumps(new_d) + \"\\n\")\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1\n",
      "2\n",
      "3\n"
     ]
    }
   ],
   "source": [
    "# combine all prompts data\n",
    "in1 = '../../minecraft/python/craftassist/text_to_tree_tool/turk_data/combined_dicts/humanbot/all_final_postprocessed.txt'\n",
    "in2 = '../../minecraft/python/craftassist/text_to_tree_tool/turk_data/combined_dicts/humanbot/other_actions/final/other_actions_postprocessed.txt'\n",
    "in3 = '../../minecraft/python/craftassist/text_to_tree_tool/turk_data/composites/humanbot_combined_postprocessed.txt'\n",
    "out_file = '../../minecraft/python/craftassist/text_to_tree_tool/turk_data/final_prompts_and_humanbot/all_humanbot_data.txt'\n",
    "\n",
    "all_text = []\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'str'>\n",
      "<class 'str'>\n"
     ]
    }
   ],
   "source": [
    "import json\n",
    "with open('../../minecraft/python/craftassist/text_to_tree_tool/turk_data/final_prompts_and_humanbot/all_humanbot_data.txt') as f, \\\n",
    "    open('../../minecraft/python/craftassist/text_to_tree_tool/turk_data/final_prompts_and_humanbot/all_humanbot_data_cp.txt', 'w') as fw:\n",
    "    for line in f.readlines():\n",
    "        line = line.strip().split(\"\\t\")\n",
    "        #print(line)\n",
    "        text, ad = line\n",
    "        print(type(ad))\n",
    "        x = json.dumps(ad)\n",
    "        print(type(x))\n",
    "        #fw.write(text + \"\\t\" + x+ \"\\n\")\n",
    "        break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('../../minecraft/python/craftassist/text_to_tree_tool/turk_data/combined_dicts/humanbot/all_final_postprocessed.txt') as f:\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "import ast\n",
    "import json\n",
    "data = []\n",
    "with open('../../minecraft/python/craftassist/text_to_tree_tool/turk_data/final_prompts_and_humanbot/all_humanbot_data.txt') as f, \\\n",
    " open('../../minecraft/python/craftassist/text_to_tree_tool/turk_data/final_prompts_and_humanbot/all_humanbot_final.json', 'w') as fw:\n",
    "    for line in f.readlines():\n",
    "        line  =  line.strip().split(\"\\t\")\n",
    "        text, ad = line\n",
    "        meh = json.loads(ad)\n",
    "        dd = ast.literal_eval(meh)\n",
    "        if dd['dialogue_type'] == 'HUMAN_GIVE_COMMAND' and len(dd['action_sequence']) == 1 and \\\n",
    "        dd['action_sequence'][0]['action_type'] == 'NOOP':\n",
    "            cnt += 1\n",
    "            dd_new = {'dialogue_type': 'NOOP'}\n",
    "            data.append([[text], dd_new])\n",
    "            continue\n",
    "        else:\n",
    "            data.append([[text], dd])\n",
    "    json.dump(data, fw)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2161\n"
     ]
    }
   ],
   "source": [
    "print(len(data))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('../../minecraft/python/craftassist/text_to_tree_tool/turk_data/final_prompts_and_humanbot/all_humanbot_final.json') as fw:\n",
    "    data_n = json.load(fw)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'list'>\n"
     ]
    }
   ],
   "source": [
    "print(type(data_n))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "dict"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "type(data_n[0][1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
